;;########################################################################
;; missdo.lsp
;; Copyright (c) 1998 by Pedro Valero (valerop@uv.es)
;; Code for Missing Data Analysis and Imputation. Can produce descriptive
;; statistics and listwise, pairwise and maximum
;; likelihood correlations for data with missing values
;;
;; Converted to ViSta Plugin by Forrest W. Young, July 2002
;;
;; The defproto was modified by FWY so that it inherits from 
;; vista-analysis-plugin-object-proto.
;; This change brings missing data into the plugin system.
;;
;; The constructor function was removed from the end of this file and
;; has been replaced by the plugin loader and constructor functions which
;; are located in the plugins directory and the plugins/missdata directory.
;;
;;########################################################################

(provide "missd0")


;;define missing-data-model object and its slot-accessors 

(defproto missing-data-model-object-proto
  '(data dialog pairwisecorrelations npairwise nvar1 nvar2 listwisecorrelations
         emcorrelations emcovariance iterations difference listwisedata
         listvariables imputed-data-random imputed-data-normal 
         data-matrix-missing missing-by-var leverages listwisedata-labels
         elementwise-convergence variables em-means mean-c-error
         iter-difs iter-logs multiple-imputed-datasets-list  initial-matrix
         patterns-missing cases-by-missing-pattern da-iterations-info data-ini)
  () vista-analysis-plugin-object-proto) ;mv-model-object-proto


#| This is the original, pre-plugin isnew method. 
   It has been replaced by the plugin isnew method which is defined below. (FWY)

(defmeth missing-data-model-object-proto :isnew (&rest args)
  (send self :model-abbrev "MD")
  (send self :dialog (fourth args))
  (send self :difference (send self :difference (sixth args)))
  (send self :iterations (send self :iterations (seventh args)))
  (send self :initial-matrix (send self :initial-matrix (select args 7)))
  (apply #'call-next-method (select args (iseq 5) '(numeric))) 
  )
|#

#| This is the plugin isnew method which replaces the pre-plugin method above (FWY)|#

(defmeth missing-data-model-object-proto :isnew 
              (title data dialog difference iterations initial-matrix)
  (send self :difference difference)
  (send self :iterations iterations)
  (send self :initial-matrix initial-matrix)
  (call-next-method title data dialog))


; 2 Definir metodos de acceso a los slots

(defmeth missing-data-model-object-proto :data
  (&optional (values nil set))
  (if set (setf (slot-value 'data) values))
  (slot-value 'data))

(defmeth missing-data-model-object-proto :data-ini
  (&optional (values nil set))
  (if set (setf (slot-value 'data-ini) values))
  (slot-value 'data-ini))

(defmeth missing-data-model-object-proto :dialog
  (&optional (values nil set))
  (if set (setf (slot-value 'dialog) values))
  (slot-value 'dialog))

(defmeth missing-data-model-object-proto :initial-matrix
  (&optional (values nil set))
"Initial matrix to start the EM algorithm. 0 means identity-matrix,
 1 means listwise correlations and 2 means pairwise correlations"
  (if set (setf (slot-value 'initial-matrix) values))
  (slot-value 'initial-matrix))

(defmeth missing-data-model-object-proto :listwisedata
  (&optional (values nil set))
  (if set (setf (slot-value 'listwisedata) values))
  (slot-value 'listwisedata))

(defmeth missing-data-model-object-proto :pairwisecorrelations
  (&optional (values nil set))
  (if set (setf (slot-value 'pairwisecorrelations) values))
  (slot-value 'pairwisecorrelations))

(defmeth missing-data-model-object-proto :listwisecorrelations
  (&optional (values nil set))
  (if set (setf (slot-value 'listwisecorrelations) values))
  (slot-value 'listwisecorrelations))

(defmeth missing-data-model-object-proto :emcorrelations
  (&optional (values nil set))
  (if set (setf (slot-value 'emcorrelations) values))
  (slot-value 'emcorrelations))

(defmeth missing-data-model-object-proto :emcovariance
  (&optional (values nil set))
  (if set (setf (slot-value 'emcovariance) values))
  (slot-value 'emcovariance))

(defmeth missing-data-model-object-proto :em-means
  (&optional (values nil set))
  (if set (setf (slot-value 'em-means) values))
  (slot-value 'em-means))

(defmeth missing-data-model-object-proto :iterations
  (&optional (values nil set))
  (if set (setf (slot-value 'iterations) values))
  (slot-value 'iterations))

(defmeth missing-data-model-object-proto :difference
  (&optional (values nil set))
  (if set (setf (slot-value 'difference) values))
  (slot-value 'difference))

(defmeth missing-data-model-object-proto :npairwise
  (&optional (values nil set))
  (if set (setf (slot-value 'npairwise) values))
  (slot-value 'npairwise))

(defmeth missing-data-model-object-proto :nvar1
  (&optional (values nil set))
  (if set (setf (slot-value 'nvar1) values))
  (slot-value 'nvar1))

(defmeth missing-data-model-object-proto :nvar2
  (&optional (values nil set))
  (if set (setf (slot-value 'nvar2) values))
  (slot-value 'nvar2))

(defmeth missing-data-model-object-proto :listvariables
  (&optional (values nil set))
  (if set (setf (slot-value 'listvariables) values))
  (slot-value 'listvariables))

(defmeth missing-data-model-object-proto :imputed-data-random
  (&optional (values nil set))
  (if set (setf (slot-value 'imputed-data-random) values))
  (slot-value 'imputed-data-random))

(defmeth missing-data-model-object-proto :imputed-data-normal
  (&optional (values nil set))
  (if set (setf (slot-value 'imputed-data-normal) values))
  (slot-value 'imputed-data-normal))

(defmeth missing-data-model-object-proto :data-matrix-missing
  (&optional (values nil set))
  (if set (setf (slot-value 'data-matrix-missing) values))
  (slot-value 'data-matrix-missing))

(defmeth missing-data-model-object-proto :missing-by-var
  (&optional (values nil set))
  (if set (setf (slot-value 'missing-by-var) values))
  (slot-value 'missing-by-var))

(defmeth missing-data-model-object-proto :leverages
  (&optional (values nil set))
  (if set (setf (slot-value 'leverages) values))
  (slot-value 'leverages))

(defmeth missing-data-model-object-proto :listwisedata-labels
  (&optional (values nil set))
  (if set (setf (slot-value 'listwisedata-labels) values))
  (slot-value 'listwisedata-labels))

(defmeth missing-data-model-object-proto :elementwise-convergence
  (&optional (values nil set))
  (if set (setf (slot-value 'elementwise-convergence) values))
  (slot-value 'elementwise-convergence))

(defmeth missing-data-model-object-proto :variables
  (&optional (values nil set))
  (if set (setf (slot-value 'variables) values))
  (slot-value 'variables))

(defmeth missing-data-model-object-proto :mean-c-error
  (&optional (values nil set))
  (if set (setf (slot-value 'mean-c-error) values))
  (slot-value 'mean-c-error))

(defmeth missing-data-model-object-proto :iter-difs
  (&optional (values nil set))
  (if set (setf (slot-value 'iter-difs) values))
  (slot-value 'iter-difs))

(defmeth missing-data-model-object-proto :iter-logs
  (&optional (values nil set))
  (if set (setf (slot-value 'iter-logs) values))
  (slot-value 'iter-logs))

(defmeth missing-data-model-object-proto :multiple-imputed-datasets-list
  (&optional (values nil set))
  (if set (setf (slot-value 'multiple-imputed-datasets-list) values))
  (slot-value 'multiple-imputed-datasets-list))

(defmeth missing-data-model-object-proto :patterns-missing
  (&optional (values nil set))
  (if set (setf (slot-value 'patterns-missing) values))
  (slot-value 'patterns-missing))

(defmeth missing-data-model-object-proto :cases-by-missing-pattern
  (&optional (values nil set))
  (if set (setf (slot-value 'cases-by-missing-pattern) values))
  (slot-value 'cases-by-missing-pattern))


(defmeth missing-data-model-object-proto :da-iterations-info
  (&optional (array nil set))
  (if set (setf (slot-value 'da-iterations-info) array))
  (slot-value 'da-iterations-info))


(defmeth missing-data-model-object-proto :options ()
  (if
   (when (send self :dialog)
         (let* (
                (result nil)
                (difference-text (send text-item-proto 
                                       :new 
                                       "Difference between parameters"))
                (iterations-text (send text-item-proto :new "Maximum number of iterations"))
                (parameters-text (send text-item-proto :new 
                                       "Initial matrix of parametters for EM estimation"))
                (difference (send edit-text-item-proto :new "0.0001"))
                (iterations (send edit-text-item-proto :new "200"))
                (Initial-matrix (send choice-item-proto :new 
                                      (list "Null covariances"
                                            "Listwise covariances"
                                            "Pairwise covariances")))
                (Ok (send modal-button-proto :new "OK" 
                          :action #'(lambda ()
                                      (list
                                       (send self :difference
                                             (read-from-string 
                                              (send difference :text)))
                                       (send self :iterations
                                             (read-from-string
                                              (send iterations :text)))
                                       (send self :initial-matrix
                                             (send initial-matrix :value))
                                       ))))
                (cancel (send modal-button-proto :new "Cancel" 
                              :action #'(lambda () 
                                         (send dialog :modal-dialog-return nil))))
                
                (dialog
                 (send modal-dialog-proto :new
                       (list (list difference-text difference) (list iterations-text iterations)
                             parameters-text
                             (list initial-matrix)
                             (list ok cancel))))
                
                (a (send dialog :default-button ok))
                (input-options (send dialog :modal-dialog))
                )
           input-options))
  (list (send self :difference) (send self :iterations) (send self :initial-matrix))
   nil))

; Esto define los analisis

(defmeth missing-data-model-object-proto :analysis ()
      (let* 
        (
         (data-ini (send current-data :active-data-matrix '(numeric)))
         (patterns 
          (patterns-missing data-ini))
         (datos-no-empty 
          (if 
           (member '0 
                   (mapcar #'(lambda (pat)
                               (sum pat))
                           patterns) :test #'=)
           (prep-datos patterns data-ini)
           (list 
            (send current-data :active-labels)
            data-ini)))
         (data (second datos-no-empty))
         (patterns (if 
                    (member '0 
                            (mapcar #'(lambda (pat)
                                        (sum pat))
                                    patterns) :test #'=)
                    (patterns-missing data)
                    patterns))
         (means-ini (mapcar #'(lambda (var)
                                (mean (non-missing var)))
                            (column-list data)))
         (variances-ini (mapcar #'(lambda (var)
                                    (variance (non-missing var)))
                                (column-list data)))
         (labels (first datos-no-empty))
         (labels (send current-data :active-labels))
         (variables (send current-data :active-variables '(numeric)))
         (missing-by-var (list-missing (copy-array data)))
         (listwisedata 
          (select data (set-difference 
                        (iseq 0 (- (array-dimension  data  0) 1)) 
                        (remove-duplicates (combine missing-by-var))) 
                  (iseq 0 (- (array-dimension  data  1) 1)))) 
         (output-em 
          (cond
            (
             (= (send self :initial-matrix) 0)
             (e-m (tobs (normalize 
                         (copy-array data-ini)) :patterns-missing patterns)
                  (send self :iterations) 
                  (send self :difference)))
            ((= (send self :initial-matrix) 1)
             (e-m (tobs (normalize
                         (copy-array data)) :patterns-missing patterns)
                  (send self :iterations) 
                  (send self :difference)
                  (listwisecorrelations listwisedata)))
            ((= (send self :initial-matrix) 2)
             (e-m (tobs (normalize
                         (copy-array data)) :patterns-missing patterns)
                  (send self :iterations) 
                  (send self :difference)
                  (first (pairwisecorrelations data missing-by-var))))
            ))
         (output-pairwisecorrelations 
          (pairwisecorrelations 
           (copy-array data)  missing-by-var))   
         (output-listwisecorrelations (listwisecorrelations  listwisedata))
         
         (listwisedata-labels  (select labels (set-difference 
                                               (iseq 0 (- (array-dimension  data  0) 1)) 
                                               (remove-duplicates (combine missing-by-var))))) 
         
         (listvariables (copy-list (list-variables-correlated 
                                    (send current-data :active-variables '(numeric)))))      
         (imputed-data nil)
         (leverages nil)
         (iter-logs (sixth output-em))
         (iter-difs (seventh output-em))
         (matrix-missingness (data-matrix-missing (copy-array data)))
         )
        ;<<<<<<< MISSD0.LSP
        
        (send self :elementwise-convergence (fifth output-em))
        (send self :patterns-missing patterns)
        (send self :cases-by-missing-pattern (eighth output-em))
        ;=======
        ;
        ;(send self :elementwise-convergence (fifth output-em))
        
        
        ;>>>>>>> 1.2
        (send self :listwisedata-labels listwisedata-labels)
        (send self :labels labels)
        (send self :missing-by-var missing-by-var)
        (send self :emcorrelations (cov-to-corr (second output-em)))
        (send self :data-matrix-missing matrix-missingness)    
        
        (send self :emcovariance 
              (corr-to-cov 
               (second output-em)
               variances-ini))
        (send self :em-means 
              (+ means-ini 
                 (* (sqrt  
                     variances-ini)
                    (first output-em))))
        (send self :data-ini data-ini)
        (send self :data data)
        (send self :listwisedata listwisedata)
        (send self :variables variables)  
        (send self :pairwisecorrelations 
              (select output-pairwisecorrelations 0))
        (send self :npairwise (select output-pairwisecorrelations 1))
        (send self :nvar1 (select output-pairwisecorrelations 2))
        (send self :nvar2 (select output-pairwisecorrelations 3))
        (send self :listwisecorrelations output-listwisecorrelations)
        (send self :listvariables listvariables)
        (setf imputed-data (send self :imputation data-ini))
        (send self :imputed-data-normal  (first imputed-data))
        (send self :imputed-data-random  (second imputed-data))
        (send self :leverages (send (regression-model (send self :imputed-data-normal) 
                                                      (normal-rand (array-dimension 
                                                                    (send self :imputed-data-normal) 0))
                                                      :print nil) :leverages))
        (send self :iter-logs (coerce (combine iter-logs) 'vector))
        (send self :iter-difs (coerce (combine iter-difs) 'vector))
      ))

                                                                                                                                                                                                                                                            
; Esto es para salvar los objetos de modelo


(defmeth missing-data-model-object-proto :save-model-template
  (data-object)
  `(missing-data-analysis
    :title  ,(send self :title)
    :name   ,(send self :name)
    :dialog nil
    :data (data ,(send data-object :name)
                :title        ,(send data-object :title)
                :variables   ',(send self :variables)
                :types       ',(send self :types)
                :labels      ',(send self :labels)
                :data        ',(combine (send self :data)))))
                
; esto crea los datos de output

(defmeth missing-data-model-object-proto :create-data 
  (&key (dialog nil)
        (pairwisecorrelations nil)
        (listwisecorrelations  nil)
        (emcorrelations nil)
        (emcovariance nil)        
        (listwisedata nil)
        
        (imputed-data nil)
        (imputed-data-random nil)
        (dummy-data nil)
        (multiple-imputation nil)
        (data-augmentation nil)
        )
  
  (if (not (eq current-object self)) (setcm self)) 
 
  (let ((creator (send *desktop* :selected-icon))
        (desires (list (list (if pairwisecorrelations 0) 
                             (if listwisecorrelations 1) 
                             (if emcorrelations 2) 
                             (if emcovariance 3) 
                             (if listwisedata 4)  
                             (if imputed-data 5) 
                             (if imputed-data-random 6) 
                             (if dummy-data 7)
                             (if multiple-imputation 8)
                             (if data-augmentation 11)
                            
                             )))
        )
    (if dialog
        (setf desires 
              (choose-subset-dialog "Select Data"
                                    '("Pairwise Correlations" 
                                                "Listwise Correlations"
                                                "Maximum Likelihood Correlations"
                                                "Maximum Likelihood Covariance"
                                                "Listwise Data"
                                                "Imputed Data"
                                               ; "Imputed Data with random variate"
                                                "Missingness Matrix"
                                                "Multiple Imputation-Data Augmentation"
                                                ;"Multiple Imputation-Random*prediction error"
                                                ;"Multiple Imputation-Closer to Random*prediction error"

                                                ; "Multiple Imputation-Propensitivity Scores"
                                                )
                                    :initial (select desires 0))))
    (when desires
          (when (member '0 (select desires 0))
                (send current-model :pairwise-correlations-data-object creator))
          (when (member '1 (select desires 0))
                (send current-model :listwise-correlations-data-object 
                      creator))
          (when (member '2 (select desires 0))
                (send current-model :e-m-correlations-data-object  creator))
          (when (member '3 (select desires 0))
                (send current-model :e-m-covariance-data-object  creator))          
          (when (member '4 (select desires 0))
                (send current-model :listwisedata-data-object  creator))         
          (when (member '5 (select desires 0))
                (send current-model :imputed-data-normal-data-object  creator))
          ;(when (member '6 (select desires 0))
           ;     (send current-model :imputed-data-random-data-object  creator))
          (when (member '6 (select desires 0))
                (send current-model :missingness-matrix-data-object  creator))
          (when (member '7 (select desires 0))  
                (send self :da-options))
          (when (member '9 (select desires 0)) 
                (send current-model :multiple-imputation3  creator 
                      :datasets 
                      (first 
                       (get-value-dialog 
                        "Number of data sets for Multiple 
Imputation-Closest to Random*prediction error:" :initial 3))))
          

          (when (member '10 (select desires 0))
                
                (send current-model :multiple-imputation4  creator
                      :datasets 
                      (first (get-value-dialog 
                              "Number of data sets Data Augmentation:" :initial 3))))
           (when (member '11 (select desires 0))
                
                (send current-model :multiple-imputation  creator 
                      :datasets 
                      (first (get-value-dialog "Number of data sets for Multiple Imputation-Propensitivity Scores :" :initial 3))))
          )
    
    (not (not desires))))

(defun list-to-matrix (correlations-list nvariables)
  (let*
    (
     (correlations-list correlations-list)
     (nvariables nvariables)
     (matrix (identity-matrix nvariables))
     (contador -1)
     )    
             (dotimes (h nvariables)
                      (dotimes (k nvariables)
                               (if (> h k)
                                   (setf (select matrix h k) 
                                         (select correlations-list                                                                  
                                                 (setf contador 
                                                       (+ 1 contador))))   
                                   )
                               (if (> h k)
                                   (setf (select matrix k h) 
                                         (select correlations-list 
                                                                 contador))   
                                   )
                               ))
    matrix))
  
(defun matrix-to-list (correlations-matrix)
  (let*
    (
     (correlations-matrix correlations-matrix)
     (nvariables (array-dimension correlations-matrix 0))
     (listcorrelations nil)    
     )
    
    (dotimes (h nvariables)
             (dotimes (k nvariables)
                      (if (> h k)
                          (setf listcorrelations 
                                (append listcorrelations                                                 
                                        (list 
                                         (select correlations-matrix h k))))
                          )
                              
                      ))
    listcorrelations))

(defmeth missing-data-model-object-proto :pairwise-correlations-data-object (creator)
  (data (strcat "PWCorr-" (send self :name))
        :created creator
        :creator-object self
        :matrices '("Pairwise Correlations")
        :shapes '("symmetric")
        :title (strcat "Pairwise Correlations for " (send self :title))
        :data (combine (send self :pairwisecorrelations))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :listwise-correlations-data-object (creator)
  (data (strcat "LWCorr-" (send self :name))
        :created creator
        :creator-object self
        :matrices '("Listwise Correlations")
        :shapes '("symmetric")
        :title (strcat "Listwise Correlations for " (send self :title))
        :data (combine (send self :listwisecorrelations))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :e-m-correlations-data-object (creator)
  (data (strcat "MLCorr-" (send self :name))
        :created creator
        :creator-object self
        :matrices '("Maximum Likelihood Correlations")
        :shapes '("symmetric")
        :title (strcat "Maximum Likelihood Correlations for " 
                       (send self :title))
        :data (combine (send self :emcorrelations))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :e-m-covariance-data-object (creator)
  (data (strcat "MLCovar-" (send self :name))
        :created creator
        :creator-object self
        :matrices '("Maximum Likelihood Covariances")
        :shapes '("symmetric")
        :title (strcat "Maximum Likelihood Covariances for " (send self :title))
        :data (combine (send self :emcovariance))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :imputed-data-normal-data-object (creator)
  (data (strcat "ImpData-" (send self :name))
        :created creator
        :creator-object self
        
        :labels (send self :labels)
        :title (strcat "Imputed data for " (send self :title))
        :data (combine (send self :imputed-data-normal))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :imputed-data-random-data-object (creator)
  (data (strcat "ImpRand-" (send self :name))
        :created creator
        :creator-object self
        
        :labels (send self :labels)
        :title (strcat "Imputed data for " (send self :title))
        :data (combine (send self :imputed-data-random))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )

(defmeth missing-data-model-object-proto :listwisedata-data-object (creator)
  (data (strcat "LWData-" (send self :name))
        :created creator
        :creator-object self
        :labels (send self :listwisedata-labels)
        :title (strcat "Listwise data for " (send self :title))
        :data (combine (send self :listwisedata))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )
(defmeth missing-data-model-object-proto :missingness-matrix-data-object (creator)
  (data (strcat "Missma-" (send self :name))
        :created creator
        :creator-object self
        :labels (send self :labels)
        :title (strcat "Missingness Matrix for " (send self :title))
        :data (combine (send self :data-matrix-missing))
        :variables (send self :active-variables '(numeric))
        :types     (send self :active-types     '(numeric)))
  )




(defmeth missing-data-model-object-proto :visualize ()
  (let
    (
     (desires nil)
     )
        
    (if (send self ::multiple-imputed-datasets-list)
         (setf desires 
              (choose-item-dialog "Select Spreadplot"
                                    '("Convergence"
                                      "Compare missing data methods"
                                      "Visualize multiple imputation series")))
        (setf desires 
              (choose-item-dialog "Select Spreadplot"
                                  '("Convergence"
                                    "Compare missing data methods"))))


    (when (= desires 0)
          (send self :spreadplot-convergence))
    (when (= desires 1)
         	(send self :viscomparecorr))
    (when (= desires 2)
         	(send self :visualize-da-imputation))
))


(defmeth missing-data-model-object-proto :report (&key (dialog nil))
  
  
  (send self :descmissing (send self :listwisedata)
               (length (row-list (send self :data)))
               (send self :data)
               (send self :imputed-data-normal)
               (send self :imputed-data-random)
               (send self :active-variables '(numeric))
               (send self :missing-by-var)
               )
   
  (send self :repcomparecorr 
        (matrix-to-list (send self :listwisecorrelations))
        (matrix-to-list (send self :pairwisecorrelations))
        (send self :npairwise ) 
        (send self :nvar1 ) 
        (send self :nvar2 ) 
        (matrix-to-list (send self :emcorrelations))
        (length (row-list (send self :listwisedata)))
        (length (row-list (send self :data)))
                  
        )
  )

